########################
# ANALYSIS: Inter Coder Agreement 
# 
# Part of replication of:
# Traditional Institutions in Africa, Past and Present
#
# Political Science Research and Methods
#
# By Clara Neupert-Wentz and Carl Müller-Crepon, 2023
#
##########################################

# Load raw data
tpi.new <- readRDS("data/intercoder_agree.rds")

#Agreement definition dummies: at least 90% of the experts agree
# Ordinal: for a 5-level ordinal variable, we measure agreement
# by the share of groups where at least 90% of the experts
# have assessments at most one level away from each other. E.g.,
# at least 90% of the experts need to rate a 4 or a 5. 
# In the tni_level_nom variable, this is the case for 82% of the observed groups . 
# If we increase the range to three levels, so we have agreement if
# 90% of the experts have assessments of at most two levels away from 
# each other, the agreement figure increases to 96%

tpi.agreements <- tpi.new %>% group_by(groupid) %>%
  summarize(agreement.any_tpi =  mean(any_tpi, na.rm = T) >= 0.9 |
              mean(any_tpi, na.rm = T) <= 0.1,
            
            agreement.tni_level_ord =  mean(tni_level_ord == 1, na.rm = T) >= 0.9 | #three point ordinal =^ dummies
              mean(tni_level_ord == 2, na.rm = T) >= 0.9| 
              mean(tni_level_ord == 3, na.rm = T) >= 0.9,
            
            agreement.tni_level_nom.brace2 =  (quantile(tni_level_nom, prob = c(0.95), #five point ordinal 2 level agreement, with 90%
                                                        type = 1, na.rm = T) -
                                                 quantile(tni_level_nom, prob = c(0.05),
                                                          type = 1, na.rm = T)) < 2,
            agreement.counceld =  mean(counceld, na.rm = T) >= 0.9 | #bodies
              mean(counceld, na.rm = T) <= 0.1,
            agreement.councking =  mean(councking, na.rm = T) >= 0.9 |
              mean(councking, na.rm = T) <= 0.1,
            agreement.assembly =  mean(assembly, na.rm = T) >= 0.9 |
              mean(assembly, na.rm = T) <= 0.1,
            agreement.dispres =  mean(dispres, na.rm = T) >= 0.9 |
              mean(dispres, na.rm = T) <= 0.1,
            agreement.rules =  mean(rules, na.rm = T) >= 0.9 |
              mean(rules, na.rm = T) <= 0.1,
            agreement.king =  mean(king, na.rm = T) >= 0.9 | #leaders
              mean(king, na.rm = T) <= 0.1,
            agreement.chief =  mean(chief, na.rm = T) >= 0.9 |
              mean(chief, na.rm = T) <= 0.1,
            agreement.headman =  mean(headman, na.rm = T) >= 0.9 |
              mean(headman, na.rm = T) <= 0.1,
            agreement.judge =  mean(judge, na.rm = T) >= 0.9 |
              mean(judge, na.rm = T) <= 0.1,
            agreement.healer =  mean(healer, na.rm = T) >= 0.9 |
              mean(healer, na.rm = T) <= 0.1,
            agreement.spiritual =  mean(spiritual, na.rm = T) >= 0.9 |
              mean(spiritual, na.rm = T) <= 0.1,
           
            agreement.formackn =  mean(formackn, na.rm = T) >= 0.9 | # Formal recognition
              mean(formackn, na.rm = T) <= 0.1,
            agreement.ties.brace2 =  (quantile(ties, prob = c(0.95), #five point ordinal 2 level agreement, with 90%
                                               type = 1, na.rm = T) -
                                        quantile(ties, prob = c(0.05),
                                                 type = 1, na.rm = T)) < 2,
            agreement.interbod =  mean(interbod, na.rm = T) >= 0.9 |# Interaction through (in)formal bodies
              mean(interbod, na.rm = T) <= 0.1,
            
            agreement.func_land =  mean(func_land, na.rm = T) >= 0.9 | # Functions
              mean(func_land, na.rm = T) <= 0.1,
            agreement.func_nrm =  mean(func_nrm, na.rm = T) >= 0.9 |
              mean(func_nrm, na.rm = T) <= 0.1,
            agreement.func_cult =  mean(func_cult, na.rm = T) >= 0.9 |
              mean(func_cult, na.rm = T) <= 0.1,
            agreement.func_fam =  mean(func_fam, na.rm = T) >= 0.9 |
              mean(func_fam, na.rm = T) <= 0.1,
            agreement.func_dr =  mean(func_dr, na.rm = T) >= 0.9 |
              mean(func_dr, na.rm = T) <= 0.1,
            agreement.func_heal =  mean(func_heal, na.rm = T) >= 0.9 |
              mean(func_heal, na.rm = T) <= 0.1,
            agreement.func_sec =  mean(func_sec, na.rm = T) >= 0.9 |
              mean(func_sec, na.rm = T) <= 0.1,
            agreement.func_spir =  mean(func_spir, na.rm = T) >= 0.9 |
              mean(func_spir, na.rm = T) <= 0.1,
            agreement.func_infra =  mean(func_infra, na.rm = T) >= 0.9 |
              mean(func_infra, na.rm = T) <= 0.1)

paste("%", c(1, 2, 3),  "%", sep = "")      

#get agreement means and turn around...

out <- paste(round(colMeans(tpi.agreements, na.rm=T), 4)*100, "%", sep = "")
names(out) <- names(colMeans(tpi.agreements, na.rm=T))

out <- t(t(c(out,
           "N Experts" = as.character(length(unique(tpi.new$lfdn))),
           "N Groups" = as.character(length(unique(tpi.new$groupid)))   
                                                    )))
# #check
# stargazer(out[-1,])

##add rownames
out <- data.frame(Variable = c("Group","Any TPI", 
                   "TPI Level (ordinal, 3 point)", "TPI Level (ordinal, 5 point)",
                   "Council of elders", "King's council", "Assembly", "Customary court", "Customary law, rules, and norms", ## Bodies
                   "King","Chief", "Headman or Bandleader", "Judge", "Healer", "Spiritual leader", ## Leaders
                   "Formal recognition",  ## Formal recognition
                   "Personal ties (ordinal, 5 point)", ## Personal ties between leaders and politicans (?) --  ordinal
                   "Interaction through (in)formal bodies", # Interaction through (in)formal bodies
                   "TPI function: Land", # Land Administration
                   "TPI function: Natural resource mgt", # Natural resource mgt
                   "TPI function: Culture", # Culture
                   "TPI function: Family matters", # Family matters
                   "TPI function: Dispute resolution", # Dispute resolution
                   "TPI function: Health", # Health
                   "TPI function: Security", # Security
                   "TPI function: Spiritual matters", # Spiritual matters
                   "TPI function: Infrastructure", # Infrastructure
                   "N Experts", "N Groups"),
                  Agreement = out)

fileConn <- file(file.path(tab.path, 
                           paste0("intercoder_agree", ".tex")))
writeLines(
stargazer(out[-1,],
          title = "Intercoder agreement",
          label = "intercoder_agree",
          summary = FALSE, rownames = FALSE,
          notes = "\\textit{Note}: Percentage of groups with at least 90\\% agreement between experts"), 
fileConn)
close(fileConn)



